<?php
include('utf8.class.php');
class ExtractKeywords {
	static private $contents;
	static private $sentences;
	static private $keywords;
	static private $words=array();	
	static private $list_words=array();	
	static private $params=array();
	static private $filter_words=array();
	static private $scores=array();
	static private $frequency=array();
	static public function Init($frequency=null)
	{
		if($frequency!=null)
		{
			reset($frequency);
			foreach($frequency as $i=>$value)
				//setcookie('frequency['.$i.']',$value, time()+5000000, '/', '.'.$_SERVER['SERVER_NAME'], 0);
			self::$frequency=$frequency;			
		}
		self::$filter_words['common'] = array();		
		self::$filter_words['unctuations']=array(',', ')', '(', '.', ' \'', '\' ', '"',
		'<', '>', ';', '!', '?', '/', '•','“','…','”','’ ',' ’',' ‘',
		'_', '[', ']', ':', '+', '=', '#','%','*','\\','»',
		'$',chr(10), chr(13), chr(9));
		$query=mysql_query('SELECT common_words FROM keywords_commons');
		mysql_query("set names utf8");
		while($common=mysql_fetch_assoc($query))
			self::$filter_words['common'][$common['common_words']]=true;
			
		$query=mysql_query('SELECT common_words FROM keywords_commons_left');
		mysql_query("set names utf8");
		while($common=mysql_fetch_assoc($query))
			self::$filter_words['common_left'][$common['common_words']]=true;

		$query=mysql_query('SELECT common_words FROM keywords_commons_right');
		mysql_query("set names utf8");
		while($common=mysql_fetch_assoc($query))
			self::$filter_words['common_right'][$common['common_words']]=true;
	}
	
	static public function  Extract()
	{
		//iterate inside the array
		$common=&self::$filter_words['common'];
		$common_left=&self::$filter_words['common_left'];
		$common_right=&self::$filter_words['common_right'];
		foreach(self::$sentences as $value=>$score ) 
		{			
			$temp_words = explode('.', str_replace('-',' ',str_replace(' ','.',$value)));	
			$temp_words = preg_replace('/ {2,}/si', ' ', $temp_words); 
			$n=count($temp_words);
			for($i=0;$i<$n;$i++)
			{
				$log_key=$temp_words[$i];
				self::$scores[$log_key]=(int)self::$scores[$log_key]+$score;
				for($j=$i+1;$j<$n;$j++)
				{
					$log_key.=' '.$temp_words[$j];
					self::$scores[$log_key]=(int)self::$scores[$log_key]+$score;
				}

				$word=$temp_words[$i];		
				if(utf8::Length($temp_words[$i])>=2 && !is_numeric(str_replace(' ','',$word )))
				{
					if(utf8::Length($temp_words[$i])>=2 && !isset($common[$word]) )
						self::$words[1][$word]=(int)self::$words[1][$word]+$score;
					if(!isset($common_left[$word]) &&	utf8::Length($temp_words[$i+1])>=2 && !is_numeric(str_replace(' ','',$temp_words[$i+1] )))	
					{
						$word=$word.' '.$temp_words[$i+1];
						if(!isset($common[$word]) && !isset($common_right[$temp_words[$i+1]]) )
							self::$words[2][$word]=(int)self::$words[2][$word]+$score;
						if(!isset($common_left[$word]) && utf8::Length($temp_words[$i+2])>=3 && !is_numeric(str_replace(' ','',$temp_words[$i+2] )))	
						{
							$word=$word.' '.$temp_words[$i+2];
							if(!isset($common[$word]) && !isset($common_right[$temp_words[$i+2]]) && !isset($common_right[$temp_words[$i+1].' '.$temp_words[$i+2]]))
								self::$words[3][$word]=(int)self::$words[3][$word]+$score;
							if(!isset($common_left[$word]) && utf8::Length($temp_words[$i+3])>=2 && !is_numeric(str_replace(' ','',$temp_words[$i+3] )))	
							{
								$word=$word.' '.$temp_words[$i+3];
								if(!isset($common[$word])  && !isset($common_right[$temp_words[$i+3]]) 
								&& !isset($common_right[$temp_words[$i+2].' '.$temp_words[$i+3]])
								&& !isset($common_right[$temp_words[$i+1].' '.$temp_words[$i+2].' '.$temp_words[$i+3]]))
									self::$words[4][$word]=(int)self::$words[4][$word]+$score;
							}
						}
					}
				}
			}		
		}	
		$occur_filtered = array();
		if (self::$words[4]!='') {

		arsort(self::$words[4]);
		foreach (self::$words[4] as $word => $occured) 
			if(self::CheckOk($word,$occured))
				$occur_filtered[$word] = $occured;
		}
		if (self::$words[3]!='') {				
		arsort(self::$words[3]);		
		foreach (self::$words[3] as $word => $occured) 
			if(self::CheckOk($word, $occured ))
				$occur_filtered[$word] = $occured;
		}

		if (self::$words[2]!='') {
		arsort(self::$words[2]);
		foreach (self::$words[2] as $word => $occured) 
			if(self::CheckOk($word, $occured ))
				$occur_filtered[$word] = $occured;
		}
		
		if (self::$words[1]!='') {
		arsort(self::$words[1]);
		foreach (self::$words[1] as $word => $occured) 
			if(self::CheckOk($word, $occured ))
				$occur_filtered[$word] = $occured;
		}
		return $occur_filtered;		
	}


	//turn the site contents into an array
	//then replace common html tags.
	static public function  AddContent($content,$score=1)
	{
		$content=strip_tags(stripslashes($content));
		$content=utf8::UnHTML(trim(self::replace_chars($content)));
	
		$sentences=explode(' . ',' '.$content.' ');
		$n=count($sentences);
		for($i=0;$i<$n;$i++)
			self::$sentences[$sentences[$i]]=(int)self::$sentences[$sentences[$i]]+$score;
	}
	static private function  replace_chars($content)
	{		
		$content = str_replace(self::$filter_words['unctuations'], ' . ',' '.$content.' ');
		 $content = preg_replace('/ {2,}/si', ' ', $content); 
		$l=utf8::Length($content);
		$isupper=false;
		$content1='';
		$words=explode(' ',$content);
		$n=count($words);
		$isupper=false;
		for($i=0;$i<$n;$i++)
		{
			if($words[$i]!='')
			{
				if(utf8::IsUpper($words[$i],0))
				{
					if($isupper)
						$content1.='-'.$words[$i];
					else
						$content1.=' '.$words[$i];
					$isupper=true;
				}else{
					$isupper=false;
					$content1.=' '.$words[$i];
				}
			}
		}
		
		$content=' '.utf8::ToLower($content1).' ';
		
		
		$query=mysql_query('SELECT word_name FROM keywords_compounds ORDER BY word_length ASC');
		mysql_query("set names utf8");
		while($compound=mysql_fetch_assoc($query))
			$content=preg_replace('/[ -]'.str_replace(' ','[ -]',$compound['word_name']).'[ -]/',' '.str_replace(' ','-',$compound['word_name']).' ',$content);
		//mysql_free_result();
		$query=mysql_query('SELECT relation_words FROM keywords_relations ORDER BY relation_length DESC');
		mysql_query("set names utf8");
		while($relation=mysql_fetch_assoc($query))
			$content=mb_ereg_replace(preg_quote(' '.$relation['relation_words'].' ','#'),' . ', $content);
		//mysql_free_result();	
		$content = str_replace(' . . ', ' . ', $content);
		
		$content=preg_replace('/ (\d+) /i',' . ',$content);
		$content = preg_replace('/ {2,}/si', ' ', $content); 
		return $content;
	}

	static private function CheckOk($word,$occured)
	{
		$n=utf8::SubStrCount($word,' ');
		if($n>5) return false;
		if(isset(self::$frequency[$n+1]) && self::$frequency[$n+1]>0)
			return self::$frequency[$n+1]<=$occured;		
		
		switch($n)
		{
			case 0:
				if(self::$scores[$word]<=4)
					return false;
			break;
			default:				
				if(self::$scores[$word]<3)
					return false;
		}		
		return true;
	}

	static public function AddKeyword($keyword)
	{


	$str=$keyword;
	$subject = htmlentities($str,ENT_QUOTES);
	$pattern = '/&([a-z])(uml|acute|circ';
	$pattern.= '|tilde|ring|elig|grave|slash|horn|cedil|th);/e';
	$replace = "'&'.strtolower('\\1').'\\2'.';'";
	$result = preg_replace($pattern, $replace, $subject);
	// convert from entities back to characters
	$htmltable = get_html_translation_table(HTML_ENTITIES);
	foreach($htmltable as $key => $value) {
	   $result = ereg_replace(addslashes($value),$key,$result);
	 }
	 $keyword=mb_strtolower($result,'utf8');




									
        //$keyword=utf8::ToLower(trim($keyword));
				//$keyword=mb_convert_case($keyword, MB_CASE_LOWER, "UTF-8"); 


		if($keyword!='')
		{

			$word=trim(str_replace(array('"','*','  '),' ',stripslashes($keyword)));
			if(strpos($keyword,'*')!==false)
			{
				if(substr_count($keyword,'*')==2)
				{
					mysql_query('INSERT INTO keywords_relations(relation_words,relation_length) values("'.mysql_escape_string($word).'", "'.utf8::Length($word).'")');
				}else if($keyword[0]=='*')
				{
					mysql_query('INSERT INTO keywords_commons_right(common_words) values("'.mysql_escape_string($word).'")');
				}else{
					mysql_query('INSERT INTO keywords_commons_left(common_words) values("'.mysql_escape_string($word).'")');
				}
			
			}else if(strpos($keyword,'"')!==false)
			{
				mysql_query('INSERT INTO keywords_compounds(word_name,word_length) values("'.mysql_escape_string($word).'", "'.utf8::Length($word).'")');
			}else{
				//mysql_query('INSERT INTO keywords_commons SET common_words="Cần"');
				mysql_query('INSERT INTO keywords_commons SET common_words="'.mysql_escape_string($word).'"');

			}
		}		
	}
}
?>

